#encoding:utf-8
'''
Created on 2015年7月12日

@author: LWD
'''

import os

from time import time
import threading
from collections import deque
from src.spider.izhihu.topic import Topic
from src.store.PMySql import PMysql

class Topic_Spider(threading.Thread):
        
    def run(self):
        '''
        @summary: 
        '''
        global mutex
        global queue
        global sset
        mysql = PMysql()
        sql_tpl = "INSERT INTO topic (id, title, description, followers, ctime) \
                       VALUES (%s, %s, %s, %s, %s)"
        # 初始化 计时 10s
        endtime = time() + 10
        while True:
            # 从队列取一个话题
            try:
                mutex.acquire()
                if len(queue) > 0:
                    (topic_id, hasChild) = queue.popleft()
                    sset.add(topic_id)
                else:
                    # 如果队列是空，等待30s, 每秒检查一次 
                    # 如果 30s后还是空，则跳出循环
                    remaining = endtime - time()
                    if remaining <= 0.0:
                        break
                    else:
                        continue
            finally:
                mutex.release()
            # 重置计时 30s
            endtime = time() + 10
            # 获取话题信息  并存储
            topic = Topic("http://www.zhihu.com/topic/"+topic_id)
            title = topic.get_topic_title()
            description = topic.get_topic_description()
            followers_num = topic.get_followers_num()
            ctime = topic.get_time()
            # 获取当前话题的父话题
            parents = topic.get_parent_topic()
            print self.name + "   " +title
            value = (topic_id, title, description, followers_num, ctime)
            mysql.insertOne(sql_tpl, value)
            # 将话题父子关系存储到数据
            values = []
            for parent in parents:
                values.append((topic_id, parent))
            mysql.insertMany(sql_tpl, values)
            # 如果话题有子话题，读取子话题，如果子话题不在队列和集合中
            # 加到队尾
            if hasChild:
                children_info = topic.get_children_topic()
                for child_info in children_info:
                    # 每次返回请求一次的所有子话题，一般是 10 个
                    # 加锁，需要访问互斥的 集合 和 队列
                    try:
                        mutex.acquire()
                        for (cchild, chasChild) in child_info:
                            # 如果 子话题不在队列  和 集合中，表示 没有被爬取  也没有待爬取
                            # 加入到队列
                            if (cchild, chasChild) not in queue and cchild not in sset:
                                queue.append((cchild, chasChild))
                    finally:
                        mutex.release()
        mysql.dispose()

# 线程之间互斥锁
mutex = threading.Lock()
# 待爬取话题队列
queue = deque()
# 已爬取的话题信息
sset = set()

if __name__ == '__main__':
    '''
    @summary: 初始添加根话题到队列，开启线程爬取话题
    '''
    print  os.path.dirname(os.path.abspath(__file__))
    
    PMysql.read_options_config()
    queue.append(("19610538", True))
    
    for i in range(2):
        p = Topic_Spider()
        p.start()
    